{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn.decomposition import  PCA\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载数据\n",
    "file_path_train = \"./data/NSL_KDD-master/KDDTrain+.csv\"\n",
    "file_path_test = \"./data/NSL_KDD-master/KDDTest+.csv\"\n",
    "train_data = pd.read_csv(file_path_train)\n",
    "test_data = pd.read_csv(file_path_test)\n",
    "data_columns =  [\"duration\",\"protocol_type\",\"service\",\"flag\",\"src_bytes\",\n",
    "            \"dst_bytes\",\"land_f\",\"wrong_fragment\",\"urgent\",\"hot\",\"num_failed_logins\",\n",
    "            \"logged_in\",\"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\n",
    "            \"num_file_creations\",\"num_shells\",\"num_access_files\",\"num_outbound_cmds\",\n",
    "            \"is_host_login\",\"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\n",
    "            \"srv_serror_rate\",\"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\n",
    "            \"diff_srv_rate\",\"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\",\n",
    "            \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\"dst_host_same_src_port_rate\",\n",
    "            \"dst_host_srv_diff_host_rate\",\"dst_host_serror_rate\",\"dst_host_srv_serror_rate\",\n",
    "            \"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\",\"labels\",\"dificulty\"]\n",
    "\n",
    "train_data.columns = data_columns\n",
    "test_data.columns = data_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land_f</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "      <th>labels</th>\n",
       "      <th>dificulty</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>private</td>\n",
       "      <td>REJ</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>neptune</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>tcp</td>\n",
       "      <td>ftp_data</td>\n",
       "      <td>SF</td>\n",
       "      <td>12983</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>normal</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>eco_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>saint</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>tcp</td>\n",
       "      <td>telnet</td>\n",
       "      <td>RSTO</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.71</td>\n",
       "      <td>mscan</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>267</td>\n",
       "      <td>14515</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>normal</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration protocol_type   service  flag  src_bytes  dst_bytes  land_f  \\\n",
       "0         0           tcp   private   REJ          0          0       0   \n",
       "1         2           tcp  ftp_data    SF      12983          0       0   \n",
       "2         0          icmp     eco_i    SF         20          0       0   \n",
       "3         1           tcp    telnet  RSTO          0         15       0   \n",
       "4         0           tcp      http    SF        267      14515       0   \n",
       "\n",
       "   wrong_fragment  urgent  hot  ...  dst_host_same_srv_rate  \\\n",
       "0               0       0    0  ...                    0.00   \n",
       "1               0       0    0  ...                    0.61   \n",
       "2               0       0    0  ...                    1.00   \n",
       "3               0       0    0  ...                    0.31   \n",
       "4               0       0    0  ...                    1.00   \n",
       "\n",
       "   dst_host_diff_srv_rate  dst_host_same_src_port_rate  \\\n",
       "0                    0.06                         0.00   \n",
       "1                    0.04                         0.61   \n",
       "2                    0.00                         1.00   \n",
       "3                    0.17                         0.03   \n",
       "4                    0.00                         0.01   \n",
       "\n",
       "   dst_host_srv_diff_host_rate  dst_host_serror_rate  \\\n",
       "0                         0.00                  0.00   \n",
       "1                         0.02                  0.00   \n",
       "2                         0.28                  0.00   \n",
       "3                         0.02                  0.00   \n",
       "4                         0.03                  0.01   \n",
       "\n",
       "   dst_host_srv_serror_rate  dst_host_rerror_rate  dst_host_srv_rerror_rate  \\\n",
       "0                       0.0                  1.00                      1.00   \n",
       "1                       0.0                  0.00                      0.00   \n",
       "2                       0.0                  0.00                      0.00   \n",
       "3                       0.0                  0.83                      0.71   \n",
       "4                       0.0                  0.00                      0.00   \n",
       "\n",
       "    labels  dificulty  \n",
       "0  neptune         21  \n",
       "1   normal         21  \n",
       "2    saint         15  \n",
       "3    mscan         11  \n",
       "4   normal         21  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 显示一部分数据\n",
    "train_data.head()\n",
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 提取标签\n",
    "X_train = train_data.drop('labels',axis=1)\n",
    "X_train = X_train.drop('dificulty', axis=1)\n",
    "labels_train = train_data['labels']\n",
    "\n",
    "labels = labels_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land_f</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>udp</td>\n",
       "      <td>other</td>\n",
       "      <td>SF</td>\n",
       "      <td>146</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>private</td>\n",
       "      <td>S0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>26</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>232</td>\n",
       "      <td>8153</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>30</td>\n",
       "      <td>255</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>199</td>\n",
       "      <td>420</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>private</td>\n",
       "      <td>REJ</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>19</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration protocol_type  service flag  src_bytes  dst_bytes  land_f  \\\n",
       "0         0           udp    other   SF        146          0       0   \n",
       "1         0           tcp  private   S0          0          0       0   \n",
       "2         0           tcp     http   SF        232       8153       0   \n",
       "3         0           tcp     http   SF        199        420       0   \n",
       "4         0           tcp  private  REJ          0          0       0   \n",
       "\n",
       "   wrong_fragment  urgent  hot  ...  dst_host_count  dst_host_srv_count  \\\n",
       "0               0       0    0  ...             255                   1   \n",
       "1               0       0    0  ...             255                  26   \n",
       "2               0       0    0  ...              30                 255   \n",
       "3               0       0    0  ...             255                 255   \n",
       "4               0       0    0  ...             255                  19   \n",
       "\n",
       "   dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n",
       "0                    0.00                    0.60   \n",
       "1                    0.10                    0.05   \n",
       "2                    1.00                    0.00   \n",
       "3                    1.00                    0.00   \n",
       "4                    0.07                    0.07   \n",
       "\n",
       "   dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "0                         0.88                         0.00   \n",
       "1                         0.00                         0.00   \n",
       "2                         0.03                         0.04   \n",
       "3                         0.00                         0.00   \n",
       "4                         0.00                         0.00   \n",
       "\n",
       "   dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "0                  0.00                      0.00                   0.0   \n",
       "1                  1.00                      1.00                   0.0   \n",
       "2                  0.03                      0.01                   0.0   \n",
       "3                  0.00                      0.00                   0.0   \n",
       "4                  0.00                      0.00                   1.0   \n",
       "\n",
       "   dst_host_srv_rerror_rate  \n",
       "0                      0.00  \n",
       "1                      0.00  \n",
       "2                      0.01  \n",
       "3                      0.00  \n",
       "4                      1.00  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 由于使用的是独热编码\n",
    "# 但是测试集中的 service 字段与训练集中的 service 缺失了7中数据因此将缺失的这七种数据加入测试集\n",
    "# 保证测试集 service 字段能够能够进行标签编码\n",
    "service = X_train['service']\n",
    "test_service = test_data['service'] \n",
    "different_service_type = np.array(list(set(service) - set(test_service)))\n",
    "np_service = np.array(list(service))\n",
    "miss_service_data_index = np.array([True if type_service in different_service_type else False for type_service in np_service])\n",
    "add_to_test_data = train_data.values[miss_service_data_index.nonzero()]\n",
    "add_to_test_data\n",
    "\n",
    "test_data = np.concatenate((test_data.values,add_to_test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.DataFrame(test_data)\n",
    "test_data.columns = data_columns\n",
    "\n",
    "X_test = test_data.drop('labels',axis=1)\n",
    "X_test = X_test.drop('dificulty', axis=1)\n",
    "labels_test = test_data['labels']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 提取特征方便进行标签编码\n",
    "test_protocol_type = X_test['protocol_type']\n",
    "test_service = X_test['service']\n",
    "test_flag = X_test['flag']\n",
    "\n",
    "protocol_type = X_train['protocol_type']\n",
    "service = X_train['service']\n",
    "flag = X_train['flag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-0.15544231, -0.17386919,  0.92101135, ..., -0.35267791,\n",
       "         1.98009391,  1.92949992],\n",
       "       [-0.15402019, -0.17386919, -0.92604007, ..., -0.35267791,\n",
       "        -0.60274276, -0.56532486],\n",
       "       [-0.15544231, -2.68617886, -1.30818864, ..., -0.35267791,\n",
       "        -0.60274276, -0.56532486],\n",
       "       ...,\n",
       "       [-0.15544231, -2.68617886,  0.98470278, ..., -0.35267791,\n",
       "        -0.31863073, -0.56532486],\n",
       "       [-0.15544231, -2.68617886,  1.87638277, ..., -0.35267791,\n",
       "        -0.60274276, -0.56532486],\n",
       "       [-0.15544231, -0.17386919, -0.79865721, ..., -0.35267791,\n",
       "         1.59266841,  1.92949992]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用标签编码对字符型离散变量进行处理\n",
    "le = preprocessing.LabelEncoder()\n",
    "enc = preprocessing.OneHotEncoder()\n",
    "lb = preprocessing.LabelBinarizer()\n",
    "\n",
    "# 对训练集进行 one-hot 编码\n",
    "X_train['protocol_type'] = le.fit_transform(protocol_type)\n",
    "X_train['service'] = le.fit_transform(service)\n",
    "X_train['flag'] = le.fit_transform(flag)\n",
    "labels_train = le.fit_transform(labels_train) + 1\n",
    "\n",
    "# 对测试集进行 one-hot 编码\n",
    "X_test['protocol_type'] = le.fit_transform(test_protocol_type)\n",
    "X_test['service'] = le.fit_transform(test_service)\n",
    "X_test['flag'] = le.fit_transform(test_flag)\n",
    "\n",
    "X = X_train\n",
    "standard_train_X = StandardScaler().fit_transform(X)\n",
    "standard_train_X\n",
    "\n",
    "test_X = X_test.to_numpy()\n",
    "standard_test_X = StandardScaler().fit_transform(test_X)\n",
    "standard_test_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Z-score标准化后的训练数据维度为(125972, 41)\n",
      "Z-score标准化后的测试数据维度为(22570, 41)\n"
     ]
    }
   ],
   "source": [
    "print('Z-score标准化后的训练数据维度为{0}'.format(standard_train_X.shape))\n",
    "print('Z-score标准化后的测试数据维度为{0}'.format(standard_test_X.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['back',\n",
       " 'buffer_overflow',\n",
       " 'ftp_write',\n",
       " 'guess_passwd',\n",
       " 'imap',\n",
       " 'ipsweep',\n",
       " 'land',\n",
       " 'loadmodule',\n",
       " 'multihop',\n",
       " 'neptune',\n",
       " 'nmap',\n",
       " 'normal',\n",
       " 'perl',\n",
       " 'phf',\n",
       " 'pod',\n",
       " 'portsweep',\n",
       " 'rootkit',\n",
       " 'satan',\n",
       " 'smurf',\n",
       " 'spy',\n",
       " 'teardrop',\n",
       " 'warezclient',\n",
       " 'warezmaster']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 记录下使用独热编码后编码所对应的攻击类型\n",
    "index_2_labels = []\n",
    "for i in range(1,24):\n",
    "    labels_ont_hot_index = (labels_train == i).nonzero()\n",
    "    label = labels[labels_ont_hot_index[0][0]]\n",
    "    index_2_labels.append(label)\n",
    "index_2_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PCA(copy=True, iterated_power='auto', n_components=41, random_state=None,\n",
       "  svd_solver='auto', tol=0.0, whiten=False)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 进行 PCA 降维\n",
    "pca = PCA(n_components=41)\n",
    "pca.fit(standard_train_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py:93: DeprecationWarning: tostring() is deprecated. Use tobytes() instead.\n",
      "  pickler.file_handle.write(chunk.tostring('C'))\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py:93: DeprecationWarning: tostring() is deprecated. Use tobytes() instead.\n",
      "  pickler.file_handle.write(chunk.tostring('C'))\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py:93: DeprecationWarning: tostring() is deprecated. Use tobytes() instead.\n",
      "  pickler.file_handle.write(chunk.tostring('C'))\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py:93: DeprecationWarning: tostring() is deprecated. Use tobytes() instead.\n",
      "  pickler.file_handle.write(chunk.tostring('C'))\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py:93: DeprecationWarning: tostring() is deprecated. Use tobytes() instead.\n",
      "  pickler.file_handle.write(chunk.tostring('C'))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['./model/pca_model.m']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.externals import joblib\n",
    "joblib.dump(pca, './model/pca_model.m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.92900974e-01 1.29485682e-01 7.04439291e-02 5.31449808e-02\n",
      " 4.85815317e-02 4.17691433e-02 3.96529926e-02 3.28079685e-02\n",
      " 2.80512201e-02 2.73051427e-02 2.64398260e-02 2.51755671e-02\n",
      " 2.50034191e-02 2.49993420e-02 2.48596994e-02 2.46763455e-02\n",
      " 2.32500385e-02 2.25602201e-02 1.92203509e-02 1.75161808e-02\n",
      " 1.55146912e-02 1.50089410e-02 1.22424414e-02 1.11758835e-02\n",
      " 1.03959990e-02 8.92111257e-03 7.85233782e-03 5.90036869e-03\n",
      " 3.48647746e-03 3.29727876e-03 2.46079181e-03 1.64211863e-03\n",
      " 1.22207487e-03 1.08234470e-03 7.21889664e-04 5.30475917e-04\n",
      " 3.59285624e-04 2.25140197e-04 1.02027978e-04 1.37640849e-05\n",
      " 1.10702605e-33]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(22570, 41)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(pca.explained_variance_ratio_)\n",
    "new_X = pca.fit_transform(standard_train_X)\n",
    "new_test_X = pca.fit_transform(standard_test_X)\n",
    "np.shape(new_test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "length_variance_contribute = np.size(pca.explained_variance_ratio_)\n",
    "contribute = 0\n",
    "index = 0\n",
    "for i in range(length_variance_contribute):\n",
    "    contribute += pca.explained_variance_ratio_[i]\n",
    "    if contribute >= 0.85:\n",
    "        index = i\n",
    "        break\n",
    "index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 4.72077694e+00, -1.69174933e+00,  4.31872684e-02, ...,\n",
       "        -1.05135331e-01,  2.78770554e-02,  3.64329555e-03],\n",
       "       [-1.43088247e+00, -1.28027867e-01, -1.30289981e-01, ...,\n",
       "         5.39221199e-02,  1.92147125e-01, -9.81759380e-02],\n",
       "       [-2.27512892e+00, -6.02057168e-01, -1.89951817e-01, ...,\n",
       "         1.08166054e+00, -3.57419089e-01, -2.55941935e-01],\n",
       "       ...,\n",
       "       [-2.63855899e-01,  1.43605810e-01,  9.27269891e-02, ...,\n",
       "         3.73489633e-01,  1.19195501e-01, -5.64377912e-02],\n",
       "       [-4.06069221e-01,  2.05861446e-01,  1.05026704e-01, ...,\n",
       "         5.20520044e-01,  7.03644987e-02, -7.20301891e-02],\n",
       "       [ 6.19125666e+00, -1.42501756e+00, -6.37627970e-02, ...,\n",
       "        -1.95611293e-01,  2.17098800e-01,  2.74905531e-01]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 产生新的特征\n",
    "new_X = new_X[:,0:index]\n",
    "new_test_X = new_test_X[:,0:index]\n",
    "np.shape(new_X)\n",
    "new_test_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22570, 17)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_test_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "对降维后的数据进行训练用时为153.26904225349426\n",
      "精度为0.9554932260795935\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "import time\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(new_X, labels_train, test_size=.3)\n",
    "\n",
    "# clf = joblib.load('./model/IDS_model_8-0.m')\n",
    "# if clf == None:\n",
    "svc = SVC(kernel='rbf', class_weight='balanced', C=0.5)\n",
    "#     c_range = np.logspace(-5, 15, 11, base=2)\n",
    "#     gamma_range = np.logspace(-9, 3, 13, base=2)\n",
    "#     param_grid = [{'kernel': ['rbf'], 'C': c_range, 'gamma': gamma_range}]\n",
    "#     grid = GridSearchCV(svc, param_grid, cv=3, n_jobs=-1)\n",
    "start = time.time()\n",
    "clf = svc.fit(x_train,y_train)\n",
    "print('对降维后的数据进行训练用时为{0}'.format(time.time() - start))\n",
    "score = clf.score(x_test, y_test)\n",
    "print('精度为%s' % score)\n",
    "    #保存模型\n",
    "#     joblib.dump(clf, './model/IDS_model_8-0.m')\n",
    "#     print('save done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(Y, labels_test, evaluate_method=1):\n",
    "    acc = 0\n",
    "    TP = 0\n",
    "    FN = 0\n",
    "    FP = 0\n",
    "    TN = 0\n",
    "    if evaluate_method == 1:\n",
    "        for predict_y, real_y in zip(Y, labels_test):\n",
    "            if predict_y == 12 and real_y == 'normal':\n",
    "                acc += 1\n",
    "            if predict_y != 12 and real_y != 'normal':\n",
    "                acc += 1\n",
    "            if predict_y == 12 and real_y != 'normal':\n",
    "                FN += 1\n",
    "            if real_y == 'normal' and predict_y !=12:\n",
    "                FP += 1\n",
    "            if predict_y != 12 and real_y != 'normal':\n",
    "                TP += 1\n",
    "            if predict_y == 12 and real_y == 'normal':\n",
    "                TN += 1\n",
    "    elif evaluate_method == 2:\n",
    "        for predict_y, real_y in zip(Y, labels_test):\n",
    "            if predict_y == 12 and real_y == 12:\n",
    "                acc += 1\n",
    "            if predict_y != 12 and real_y != 12:\n",
    "                acc += 1\n",
    "            if predict_y == 12 and real_y != 12:\n",
    "                FN += 1\n",
    "            if real_y == 12 and predict_y !=12:\n",
    "                FP += 1\n",
    "            if predict_y != 12 and real_y != 12:\n",
    "                TP += 1\n",
    "            if predict_y == 12 and real_y == 12:\n",
    "                TN += 1\n",
    "\n",
    "    precent_acc = acc / len(labels_test)\n",
    "    print(TP, FN)\n",
    "    print(FP, TN)\n",
    "    TPR = TP / (TP + FN)\n",
    "    FPR = FP / (FP + TN)\n",
    "    print('acc radio %s ' % precent_acc)\n",
    "    print('TPR is %s' % TPR)\n",
    "    print('FPR is %s' % FPR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8569 4270\n",
      "4027 5704\n",
      "acc radio 0.6323881258307488 \n",
      "TPR is 0.6674195809642496\n",
      "FPR is 0.41383208303360397\n"
     ]
    }
   ],
   "source": [
    "# 对 NSL-KDD-test_set 进行模型评估\n",
    "Y = clf.predict(new_test_X)\n",
    "evaluate(Y, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17500 40\n",
      "1613 18639\n",
      "acc radio 0.9562605842506351 \n",
      "TPR is 0.9977194982896237\n",
      "FPR is 0.07964645467114359\n"
     ]
    }
   ],
   "source": [
    "# 对 NSL-KDD-30%-train_set 进行模型评估\n",
    "Y = clf.predict(x_test)\n",
    "evaluate(Y, y_test, evaluate_method=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_labels = {'DOS':['back','neptune','smurf','teardrop','land','pod','apache2','mailbomd','processtable'],\n",
    "              'Probe':['satan','portsweep','ipsweep','nmap','mscan','saint'],\n",
    "              'R2L':['warezmaster','ftp_write','guess_passwd','imap','multihop','phf','spy','warezclient','sendmail','named','snmpgetattack','snmpguess','xlock','xsnoop','worm'],\n",
    "              'U2R':['rootkit','buffer_overflow','loadmodule','perl','httptunnel','ps','sqlattack','xterm'],\n",
    "              'NORMAL':['normal']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['DOS', 'Probe', 'R2L', 'U2R', 'NORMAL'])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "types_attack = all_labels.keys()\n",
    "types_attack"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
